package server;


import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import models.Category;
import models.Domain;
import models.Inlink;
import models.Keyword;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;
import com.googlecode.objectify.Key;
import com.googlecode.objectify.Objectify;
import com.googlecode.objectify.ObjectifyService;
import com.googlecode.objectify.Query;

import controller.Utils;

public class Update {
    
    private URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
    
    private Domain domain = new Domain();
    
    private TreeMap<String, String> check = new TreeMap<String, String>();
    
    private Future fDomain = null;
    
    private Future fWhois = null;
    
    private Future fIpAddress = null;
    
    private Future fAlexa = null;
    
    private Future fGoogleRank = null;
    
    private Future fAgeDomain = null;
    
    private Future fValidator = null;
    
    private Future fGoogleService = null;
    
    private Future fMarkosweb = null;
    
    private Future fWot = null;
    
    private String content = "";
    
    private String alexaContent = "";
    
    private List<String> googleAnalyticSites = new ArrayList<String>();
    
    private List<String> googleAdsenceSites = new ArrayList<String>();
    
    private List<String> siteSameIp = new ArrayList<String>();
    
    private Long time1 = new Date().getTime();
    
    public Update(Domain domain) {
        this.domain = domain;
        //Load content from urls
        loadUrlDomain();
        
        loadUrlWhois();
        
        loadUrlIpAddress();
        
        loadUrlAlexa();
        
        loadUrlGoogleRank();
        
        loadUrlAgeDomain();

        loadMarkosweb();
        
        loadWot();
        
        //Get content form url loaded
        fDomainService();
        
        getWhois();
        
        getIpAddress();
        
        fAlexaService();
        
        getGooglePageRank();
        
        getAgeDomain();
        
        getMarkosweb();
        
        getWot();
        
        try {
			ObjectifyService.register(Domain.class);
		} catch (Exception e) {
			// TODO: handle exception
		}
		
		Objectify objectify = ObjectifyService.begin();
		this.domain.setDate(new Date());
		PutCache.putCacheList(this.domain.getDomain(), "updates",19);
		
		objectify.put(this.domain);
    }    

    /************************************* Load URL *************************************/
    private void loadMarkosweb() {
    	try {
            URL urlMarkosweb = new URL("http://www.markosweb.com./www/" + domain.getDomain());
            fMarkosweb = fetcher.fetchAsync(urlMarkosweb);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fMarkosweb", "Can't run Markosweb");
        }
    }
    
    private void loadWot() {
    	try {
            URL urlWot = new URL("http://api.mywot.com/0.4/public_query2?url=http://" + domain.getDomain());
            fWot = fetcher.fetchAsync(urlWot);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWot", "Can't run WOT");
        }
    }
    
    private void loadUrlDomain() {
        try {
            URL urlDomain = new URL("http://" + domain.getDomain());
            fDomain = fetcher.fetchAsync(urlDomain);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadUrlWhois() {
        try {
        	URL urlWhois = new URL("http://whois-server.net/?query="+Utils.returnSubDomainTo(domain.getDomain()));
            fWhois = fetcher.fetchAsync(urlWhois);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Can't run fWhois");
        }
    }
    
    private void loadUrlIpAddress() {
        try {
            URL urlIpAddress = new URL("http://www.nearby.org.uk/api/dig.php?l=" + domain.getDomain());
            fIpAddress = fetcher.fetchAsync(urlIpAddress);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fIpAddress", "Can't run fIpAddress");
        }
    }
    
    private void loadUrlAlexa() {
        try {
            URL urlAlexa = new URL("http://www.alexa.com/siteinfo/" + domain.getDomain());
            fAlexa = fetcher.fetchAsync(urlAlexa);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't run fAlexa");
        }
    }
    
    private void loadUrlGoogleRank() {
        try {
            URL urlGoogleRank = new URL("http://gpagerank.net/?url=" + domain.getDomain());
            fGoogleRank = fetcher.fetchAsync(urlGoogleRank);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fGoogleRank", "Can't run fGoogleRank");
        }
    }
    
    private void loadUrlAgeDomain() {
        try {
            URL urlAgeDomain = new URL("http://www.seologs.com/dns/domain-check.html");
            HTTPRequest request = new HTTPRequest(urlAgeDomain, HTTPMethod.POST);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            String params = "query=" + domain.getDomain() + "&output=cdt";
            request.setPayload(params.getBytes());
            fAgeDomain = fetcher.fetchAsync(request);

        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAgeDomain", "Can't run fAgeDomain");
        }
    }
    

    /************************************* Get content *************************************/
    private void fDomainService() {
        try {
        	HTTPResponse response = (HTTPResponse) fDomain.get();
            
            // if redirects are followed, this returns the final URL we are redirected to
            URL finalUrl = response.getFinalUrl();
            
            String charset="utf-8";
            
            this.content = new String(response.getContent(),charset);
    			
            Document doc = Jsoup.parse(this.content);
            
            int index = doc.select("meta").toString().toLowerCase()
			.indexOf("charset=");
			if (index != -1) {
				charset = doc.select("meta").toString().substring(index + 8)
						.replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "")
						.replaceAll("[\\'\\\"]+", "");
			}
			
            this.domain.setCharset(charset);
            
			this.content = new String(response.getContent(),charset);
			
    		getServerInfo(response);
            
            getSiteInfo();
            getGoogleAnalytic();
            getGoogleAdsense();
            getJavascriptLibrary();
            getInlink();
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't response fDomain");
        }
    }
    
    private void fAlexaService() {
        try {
        	HTTPResponse response = (HTTPResponse) fAlexa.get();
            
            this.alexaContent = new String(response.getContent(),"utf-8");
            
            getAlexaRank();
            getPopularKeyword();
//            getCategory();
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getPopularKeyword() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            Elements elements = doc.select("#top-keywords-from-search");
            String keyword = "";
            if(elements.size() > 0)
			{
				for (Element element : elements.first().select("u")) {
					keyword+=element.text()+", ";
				}
			}
            domain.setPopularKeyword(new Text(keyword));
            
            Objectify objectify = ObjectifyService.begin();
            
            if(!keyword.equals(""))
            {
            	String[] key = keyword.split(",");
            	for (int i = 0; i < key.length; i++) {
					String string = key[i].trim();
					
					try {
                    	Keyword inlink = objectify.get(new Key<Keyword>(Keyword.class, string));
                    	inlink.addDomain(this.domain.getDomain());
                    	objectify.put(inlink);
					} catch (Exception e) {
//									e.printStackTrace();
						Keyword inlink = new Keyword();
						inlink.setKeyword(string);
						inlink.addDomain(this.domain.getDomain());
						objectify.put(inlink);
					}
				}
            }
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getCategory() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            Elements elements = doc.select("#siteSentences + div").select("a");
            String cate = "";
            if(elements.size() > 0)
			{
            	int i=0;
            	for (Element element : elements) {
    				if(i>4) break; 
    				String categoryName = element.text();
    				cate+=categoryName+", ";
    				
    				try {
    	    			ObjectifyService.register(Category.class);
    	    		} catch (Exception e) {
    	    			// TODO: handle exception
    	    		}
    	    		Objectify objectify = ObjectifyService.begin();
    				
    	    		try {
    	    			Category cate1 = objectify.get(new Key<Category>(Category.class, categoryName));
    	    			cate1.add(this.domain.getDomain());
    	    			objectify.put(cate1);
					} catch (Exception e) {
						Category category = new Category();
						category.setCate(categoryName);
						category.add(this.domain.getDomain());
						objectify.put(category);
					}
    	    		
    				i++;
            	}
			}
            domain.setCategory(new Text(cate));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getServerInfo(HTTPResponse response) {
    	try {
            //Get webServer
            String webServer = "";
            String programmingLanguage = "";
            for (int i = 0; i < response.getHeaders().size(); i++) {
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("server")) {
                    webServer = response.getHeaders().get(i).getValue();
                }
                
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("x-powered-by")) {
                    programmingLanguage = response.getHeaders().get(i).getValue();
                }
                //System.out.println(response.getHeaders().get(i).getName() + " - " + response.getHeaders().get(i).getValue());
            }
            
            this.domain.setServer(webServer);
            this.domain.setLanguage(programmingLanguage);
            
        } catch (Exception e) {
            this.check.put("server", domain.getDomain() + " - Webserver not found");
        }
    }
    
    private void getSiteInfo() {
        try {
            Document doc = Jsoup.parse(this.content);
            
            Long time2 = new Date().getTime();
            
            String speed = String.valueOf(((float)(time2-time1))/1000);
            
            this.domain.setSpeed(speed);
            
            String title = "";
            String description = "";
            String keywords  = "";
            String des = "";
            int mkey = 0;
            int mdes = 0;
            try {
                title = doc.select("title").first().text();
            } catch (Exception e) {
            	e.printStackTrace();
                this.check.put("title", "Can't get info title");
            }

            try {
                keywords = doc.select("meta[name=keywords]").first().attr("content");
                mkey = 1;
            } catch (Exception e) {
                try {
                    keywords = doc.select("meta[http-equiv=keywords]").first().attr("content");
                } catch (Exception e2) {
                    this.check.put("keywords", "Can't get info keywords");
                }        
            }
            
            try {
                description = doc.select("meta[name=description]").first().attr("content");
                mdes = 1;
            } catch (Exception e) {
                try {
                    description = doc.select("meta[http-equiv=description]").first().attr("content");
                } catch (Exception e2) {
                    if(doc.select("body").first().text().length() > 260)
                    {
                        description = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "")+" ...";
                    } else {
                        description = doc.select("body").first().text();
                    }
                }
            }
            des = doc.select("body p").text();
            if(des.length()<100)
            	des = doc.select("body").first().text();
            if(des.length()>250)
            	des = des.substring(0,240);
            
            
            if(description.length()>260)
            {
                description = description.substring(0,250);
            }
            
            Elements element = doc.select("h1");
				this.domain.setH1(element.size());
			element = doc.select("h2");	
				this.domain.setH2(element.size());
			element = doc.select("h3");	
				this.domain.setH3(element.size());
			element = doc.select("h4");	
				this.domain.setH4(element.size());
			element = doc.select("h5");	
				this.domain.setH5(element.size());
            
			element = doc.select("img");
			
			int count = 0;
			for(int i=0;i<element.size();i++)
			{
				if(element.get(i).attr("alt") != null && !element.get(i).attr("alt").equals(""))
				{
					count++;
				}
			}
			this.domain.setImgAlt(count);
			this.domain.setImg(element.size());
			element = doc.select("embed");
				this.domain.setFlash(element.size());
			element = doc.select("frame");
				this.domain.setFrame(element.size());
			
			this.domain.setSize(String.valueOf(doc.html().length()));
			this.domain.setCheckKey(mkey);
			this.domain.setCheckDes(mdes);
            this.domain.setTitle(title);
            this.domain.setKeywords(new Text(keywords));
            this.domain.setDescription(new Text(description));
            this.domain.setDes(new Text(des));
            
            try {
    			ObjectifyService.register(Keyword.class);
    		} catch (Exception e) {
    			// TODO: handle exception
    		}
    		Objectify objectify = ObjectifyService.begin();
            
            if(!keywords.equals(""))
            {
            	String[] key = keywords.split(",");
            	for (int i = 0; i < key.length; i++) {
					String string = key[i].trim();
					
					try {
                    	Keyword inlink = objectify.get(new Key<Keyword>(Keyword.class, string));
                    	inlink.addDomain(this.domain.getDomain());
                    	objectify.put(inlink);
					} catch (Exception e) {
//									e.printStackTrace();
						Keyword inlink = new Keyword();
						inlink.setKeyword(string);
						inlink.addDomain(this.domain.getDomain());
						objectify.put(inlink);
					}
				}
            }
            
            
        } catch (Exception e) {
        	e.printStackTrace();
            this.check.put("siteinfo", "Can't connect to domain " + domain.getDomain());
        }
    }
    
    private void getGoogleAnalytic() {
        int begin = content.indexOf("_gaq.push(['_setAccount', '");
        String analyticAccount = "";
        if(begin>0)
        {
            begin+="_gaq.push(['_setAccount', '".length();
            int end =  content.indexOf("'",begin);
            analyticAccount = content.substring(begin, end);
            begin=analyticAccount.lastIndexOf("-")-1;
            
            analyticAccount = analyticAccount.substring(0, begin);
        }
        else
        {
             begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
            if(begin>0)
            {
                begin+="_gaq.push([\"_setAccount\", \"".length();
                int end =  content.indexOf("\"",begin);
                analyticAccount = content.substring(begin, end);
                
            } else {
                begin = content.indexOf("_uacct");
                if(begin > 0)
                {
                    begin+="_uacct".length();
                    analyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
                    begin=analyticAccount.lastIndexOf("-");
                    
                    analyticAccount = analyticAccount.substring(0, begin);
                } else {
                    Pattern p = Pattern.compile("UA-[0-9]+");

                    Matcher m = p.matcher(content);
                    while (m.find()) {
                        analyticAccount = m.group();
                    }
                }
            }
        }
        
        this.domain.setGoogleAnalytic(analyticAccount);
    }
    
    private void getGoogleAdsense() {
        int begin = content.indexOf("google_ad_client = \"");
        String adsAccount = "";
        if( begin > 0 )
        {
            begin+="google_ad_client = \"', '".length();
            int end =  content.indexOf("\"",begin);
            adsAccount = content.substring(begin, end);
        }
        
        this.domain.setGoogleAdsense(adsAccount);
    }
    
    private void getJavascriptLibrary() {
        List<String> jsLib = new ArrayList<String>();
        if(content.indexOf("jquery")>0)
            jsLib.add("jquery");
        
        if(content.indexOf("prototype.js")>0)
            jsLib.add("prototype");
        
        if(content.indexOf("extjs")>0)
            jsLib.add("extjs");
        
        if(content.indexOf("yui")>0)
            jsLib.add("yui");
        
        if(content.indexOf("dojo")>0)
            jsLib.add("dojo");
        
        if(content.indexOf("gwt")>0)
            jsLib.add("gwt");

        if(content.indexOf("easeljs")>0)
            jsLib.add("easeljs");
        
        this.domain.setJavascript(StringHelper.listToString(jsLib, ", "));
    }
    
    private void getWhois() {
        try {
            
            HTTPResponse response = (HTTPResponse) fWhois.get();
            
            Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
            
            String whois = Utils.cleanWhois(doc.select("td[height=40]").html());
            
            Pattern p = Pattern.compile("[^ :]+@[^ ]+");

	        Matcher m = p.matcher(whois.replaceAll("\\<.*?\\>", " "));

	        ArrayList<String> email = new ArrayList<String>();
	        String listEmail = "";
	        while (m.find()) {
	        	int k=0;
	        	for (int i = 0; i < email.size(); i++) {
					if(email.get(i).equals(m.group().replaceAll("[()]+", "")))
					{
						k=1;
					}
				}
	        	if(k==0)
	        	{
	        		email.add(m.group().replaceAll("[()]+", ""));
	        		listEmail += m.group().replaceAll("[()]+", "")+" , ";
	        	}
	        }
	        
//	        String dig = doc.select("#dig").html();
//	        String ns = doc.select("#ns").html();
//	        String wip = doc.select("#wip").html();
	        
//	        this.domain.setDsn(new Text(dig));
//	        this.domain.setNs(new Text(ns));
//	        this.domain.setWhoisIp(new Text(wip));
            this.domain.setEmail(listEmail);
            this.domain.setWhois(new Text(whois));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Error when get value whois");
        }
    }
    
    private void getMarkosweb() {
        try {
            
            HTTPResponse response = (HTTPResponse) fMarkosweb.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            
            Text linkAnalysis = new Text("");
            Text indexPage = new Text("");
            Text backwardLink = new Text("");
            Text htmlValidation = new Text("");
            
            Elements elements = doc.select(".column script");
            
            if(elements.size()>0)
            {
            	for (int i = 0; i < elements.size(); i++) {
					if(i==0)
						linkAnalysis=new Text(elements.get(i).toString());
					if(i==1)
						indexPage=new Text(elements.get(i).toString());
					if(i==2)
						backwardLink=new Text(elements.get(i).toString());
					if(i==3)
						htmlValidation=new Text(elements.get(i).toString());
				}
            }
            
            this.domain.setLinkAnalysis(linkAnalysis);
            this.domain.setIndexPage(indexPage);
            this.domain.setBackwardLink(backwardLink);
            this.domain.setHtmlValidation(htmlValidation);
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Error when get value Markosweb");
        }
    }
    
    private void getWot() {
    	try {
    		HTTPResponse response = (HTTPResponse) fWot.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            
            Elements checks = doc.select("application");
			 
			if(checks.size() > 0 && checks.size() == 4)
			{
				String check = doc.select("application").last().attr("r").trim();
				
				this.domain.setWot(check);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
    }
    
    private void getIpAddress() {
        try {
        	HTTPResponse response = (HTTPResponse) fIpAddress.get();
            
            String ip = new String(response.getContent());
            
            this.domain.setIp(ip);
            
            try {
				String content = Utils.GetHtml("http://whois-server.net/?query="+ip, "utf-8");
				Document document = Jsoup.parse(content);
				String whoisIp = Utils.cleanWhois(document.select("td[height=40]").html());
				
				this.domain.setWhoisIp(new Text(whoisIp));
			} catch (Exception e) {
				e.printStackTrace();
			}
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("ipAddress", "Can't get ipAddress for this domain " + domain.getDomain());
        }
    }
    
    private void getAlexaRank() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            String strAlexaRank = doc.select("#siteStats").select("tbody tr td div").first().text().trim().replace(",","");
            int alexaRank = 0;
            
            try {
                if(strAlexaRank != null && !strAlexaRank.isEmpty())
                {
                    alexaRank = Integer.parseInt(strAlexaRank.replaceAll("[^0-9]+","")) ;
                }
            } catch (Exception e) {
            	e.printStackTrace();
                check.put("alexaGlobeRank", "Can't get position alexa rank of domain " + domain.getDomain());
            }
            
            // visitor
			String visit = "0";
			if(doc.select(".visitors_percent").select(".avg").size() > 0)
			{
				visit = doc.select(".visitors_percent").select(".avg").last().text().trim();
			}
            
			this.domain.setVisit(visit);
			
            this.domain.setAlexa(alexaRank);
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getGooglePageRank() {
        try {
            HTTPResponse response = (HTTPResponse) fGoogleRank.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            String googlePageRank = "0";
            String google = doc.select(".show").select("span").first().select("font").text().replaceAll("[^0-9]", "");
            try {
                if(google!= null && !google.isEmpty())
                {
                    googlePageRank = google;
                }
            } catch (Exception e) {
            	e.printStackTrace();
                check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
            }
            
            this.domain.setGooglepr(googlePageRank);
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
        }
    }
    
    private void getAgeDomain() {
        try {
            HTTPResponse response = (HTTPResponse) fAgeDomain.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            String age = doc.select("div[class=KonaBody]").select("p").text();
               String[] ageSplit = age.split("(approximately)");

               if(ageSplit.length ==2)
               {
                   this.domain.setAge(ageSplit[1].replaceAll("[):]+", ""));
               }
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("age", "Can't get age of domain " + domain.getDomain());
        }
    }
    
    private void getInlink() {
        try {
            Document doc = Jsoup.parse(this.content);
            Elements link = doc.select("a");
            
            ArrayList<String> listLink = new ArrayList<String>();
            String check = null;
            
//            try {
//    			ObjectifyService.register(Inlink.class);
//    		} catch (Exception e) {
//    			// TODO: handle exception
//    		}
//    		Objectify objectify = ObjectifyService.begin();
            
            for(int i=0;i<link.size();i++)
            {
                check = link.get(i).attr("rel"); 
                if(Pattern.matches("^http[\\w\\W]+", link.get(i).attr("href")))
                {
                    if( check != null && !check.equals("nofollow") || check == null)
                    {
                        Pattern p = Pattern.compile("^http[^\\/]+[\\/]{2}[^\\/]+");
    
                        Matcher m = p.matcher(link.get(i).attr("href"));
                        while (m.find()) {
                            if(!listLink.contains(m.group())&&!Utils.fomatDomain(m.group()).equals(this.domain.getDomain()))
                            {
                            	if(Utils.checkDomain(Utils.fomatDomain(m.group())))
                            	{
	                                listLink.add(m.group());
	                                
//	                                try {
//	                                	Inlink inlink = objectify.get(new Key<Inlink>(Inlink.class, Utils.fomatDomain(m.group())));
//	                                	inlink.addInLink(this.domain.getDomain());
//	                                	objectify.put(inlink);
//									} catch (Exception e) {
//	//									e.printStackTrace();
//										Inlink inlink = new Inlink();
//										inlink.setDomain(Utils.fomatDomain(m.group()));
//										inlink.addInLink(this.domain.getDomain());
//										objectify.put(inlink);
//									}
                            	}
                            }
                        }
                    }
                }
            }
            
            this.domain.setOutLink(new Text(StringHelper.listToString(listLink, ",")));
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("inlink", "Can't get inlink of domain " + domain.getDomain());
        }
    } 
}
